

import sys

import FileUtil

import ext
import Reg

# This is main python script for Tokenizer.


# get parsers by language
def get_parsers_by_language(language):
    if language not in Reg.Language_Parsers:
        return (False, F"""language {language} is not supported!""")
    
    parsers = []

    parser_ids = Reg.Language_Parsers[language]()
    for id in parser_ids:
        if id in Reg.Token_Parsers:
            parsers.append(Reg.Token_Parsers[id])
        else:
            print(f"""parser: {id} is not matched!""")
    
    return (True, parsers)


# Tokenizer main process
# 
def tokenizer(content, language):
    f_offset = 0  # offset in file
    line = 0      # current line number
    l_offset = 0  # offset in line
    
    # get all token parser defined in current language
    (ret, matchers) = get_parsers_by_language(language)
    #print(matchers)

    # result tokens
    tokens = []

    while f_offset >= 0 and f_offset < len(content):
        # look a char 
        c = content[f_offset:f_offset+1]

        # try to process invisible char
        # TODO !!!!!
        # \\ is valid for all language??????
        if c == " " or c == "\\" or c == "\t" or c == "\r" or c == "\n":
            f_offset += 1
            if c == "\n":
                line += 1
                l_offset = 0
            else:
                l_offset += 1
            continue


        # attempt to match a token parser
        matcher = None
        for m in matchers:
            # find a match parse, then stop to find
            if 'condition' in m and m['condition'](f_offset, content):
                matcher = m
                break
        
        # if not find a match parser, then report error!
        if matcher is None:
            return (False, F"""line: {line} offset: {l_offset} match fail!""")
        
        # try to parse token by matched parser
        (re, token, fn_offset, tok_type) = matcher['parser'](content, f_offset)
        if re != True:
            return (False, F"""line: {line} offset: {l_offset} parse fail!""")

        # finish token info
        tok = {}
        tok['text'] = token
        tok['line'] = line
        tok['column'] = l_offset
        tok['offset'] = f_offset
        tok['type'] = tok_type
  
        matcher = None
        # post token process
        for m in matchers:
            # find a post match parser, then stop to find
            if 'post_condition' in m and tok['text'] in m['post_condition']:
                matcher = m
                break
        
        # report error
        if matcher is not None:
            # do post token parser     
            (re, tok) = matcher['post_parser'](tok)
            if re != True:
                return (False, F"""line: {line} offset: {l_offset} token:{tok['text']}  post parse fail!""")
        
        # save token parsed
        tokens.append(tok)
        
        # visit text between f_offset and fn_offset
        # refix line number and offset number
        for i in range(f_offset, fn_offset):
            c = content[i:i+1]
            
            # wether is newline flag
            # TODO !!!
            if c == "\n":
                line += 1
                l_offset = -1
            f_offset += 1
            l_offset += 1

        # reset f_offset var
        f_offset = fn_offset

    return (True, tokens) 



# Source language recognition
# fname: file name with full path
def language_recognition(fname):
    # language recognition by extension
    ext = fname.split('.')[-1]

    if ext in Reg.Language_Recognition:
        return (True, Reg.Language_Recognition[ext])
    return (False, F"""unknow language extension: {ext}""")


# Main Process
def main(fname, oname):
    # Recognize language
    (ret, language) = language_recognition(fname)
    if ret == False:
        print("Recognize language fail!", language)
        exit(-1)

    # read all lines in input_file
    (ret, content) = FileUtil.load_file_content(fname)
    if ret == False:
        print("Read input file fail!", content)
        exit(-1)
    
    # Tokenize process
    return tokenizer(content, language)


if __name__ == "__main__":
    # input parameter test
    if len(sys.argv) != 3:
        print("python Tokenizer.py input_file output_file")
        exit(-1)

    fname = sys.argv[1]
    oname = sys.argv[2]

    (ret, tokens) = main(fname, oname)
    print(ret)
    print(tokens)
    


